from math import sqrt
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import plot_precision_recall_curve
from sklearn.metrics import plot_roc_curve

# 导入乳腺癌数据集
X, y = datasets.load_breast_cancer(return_X_y=True)

# 将样本分为10个子集以供交叉验证
cv = ShuffleSplit(n_splits=10, test_size=0.35, random_state=0)

# 创建支持向量机模型
clf1 = svm.SVC(kernel='linear', C=1, random_state=42)
# 创建K近邻模型
clf2 = KNeighborsClassifier()

print('\n============== SVM ==============')

# 输出10折交叉验证的查准率
precision = cross_val_score(
    clf1, X, y, cv=cv, scoring='precision'
)
print('查准率：\n', precision, '\n平均值：', precision.mean())

# 输出10折交叉验证的查全率
recall = cross_val_score(
    clf1, X, y, cv=cv, scoring='recall'
)
print('查全率：\n', recall, '\n平均值：', recall.mean())

# 输出10折交叉验证的F1-score
f1 = cross_val_score(
    clf1, X, y, cv=cv, scoring='f1'
)
print('F1-score：\n', f1, '\n平均值：', f1.mean())

# 输出10折交叉验证的错误率，以进行paired t-tests
accuracy = cross_val_score(
    clf1, X, y, cv=cv, scoring='accuracy'
)
inaccuracy_SVM = 1 - accuracy
print('错误率：\n', inaccuracy_SVM)

print('\n============== KNN ==============')

# 输出10折交叉验证的查准率
precision = cross_val_score(
    clf2, X, y, cv=cv, scoring='precision'
)
print('查准率：\n', precision, '\n平均值：', precision.mean())

# 输出10折交叉验证的查全率
recall = cross_val_score(
    clf2, X, y, cv=cv, scoring='recall'
)
print('查全率：\n', recall, '\n平均值：', recall.mean())

# 输出10折交叉验证的F1-score
f1 = cross_val_score(
    clf2, X, y, cv=cv, scoring='f1'
)
print('F1-score：\n', f1, '\n平均值：', f1.mean())

# 输出10折交叉验证的错误率，以进行paired t-tests
accuracy = cross_val_score(
    clf2, X, y, cv=cv, scoring='accuracy'
)
inaccuracy_KNN = 1 - accuracy
print('错误率：\n', inaccuracy_KNN)

print('\n============== paired t-tests ==============')
Delta = inaccuracy_SVM - inaccuracy_KNN # 对每对结果求差
mu = Delta.mean() # 计算均值
sigma = Delta.std() # 计算标准差
t = abs(sqrt(10) * mu / sigma)
print("变量的值为：", t)


# 单独划分一份训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.35
)

# 创建并训练一个支持向量机模型
clf_SVM = svm.SVC(kernel='linear', C=1, random_state=42).fit(X_train, y_train)
# 创建并训练一个K近邻模型
clf_KNN = KNeighborsClassifier().fit(X_train, y_train)

# 绘制混淆矩阵
SVM_CM = plot_confusion_matrix(clf_SVM, X_test, y_test)
SVM_CM.figure_.suptitle("SVM confusion matrix")
plt.savefig('./SVM confusion matrix.png')
# plt.show()
KNN_CM = plot_confusion_matrix(clf_KNN, X_test, y_test)
KNN_CM.figure_.suptitle("KNN confusion matrix")
plt.savefig('./KNN confusion matrix.png')
# plt.show()

# 绘制P-R曲线
svm_disp = plot_precision_recall_curve(clf_SVM, X_test, y_test)
knn_disp = plot_precision_recall_curve(clf_KNN, X_test, y_test, ax=svm_disp.ax_)
knn_disp.figure_.suptitle("P-R curve comparison")
plt.savefig('./P-R curve comparison.png')
# plt.show()

# 绘制ROC曲线
svm_disp = plot_roc_curve(clf_SVM, X_test, y_test)
knn_disp = plot_roc_curve(clf_KNN, X_test, y_test, ax=svm_disp.ax_)
knn_disp.figure_.suptitle("ROC curve comparison")
plt.savefig('./ROC curve comparison.png')
# plt.show()
